{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.6585406 , -0.7525452 , -0.69864213], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Pendulum-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAlXklEQVR4nO3de3DU5aH/8c93c9kQwm5IIFkjCaByhFTACghbzhnbkhI1tVVxjmUYpUp1xEBBHFvxgpfpTDj4a1UqYDttwXNG5RyseEFR0yDhqOEW4RgRo1UkUdiEi9kNl2wu+/z+8LDHVbQJZLNPwvs1szPm+312n2e/wr7Z3e9uHGOMEQAAFnIlegEAAHwTIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsFbCIrVs2TINGzZMaWlpmjhxorZu3ZqopQAALJWQSP3nf/6nFixYoPvuu09vv/22xo4dq+LiYjU2NiZiOQAASzmJ+ILZiRMnasKECXrsscckSZFIRPn5+Zo7d67uvPPOnl4OAMBSyT09YWtrq6qrq7Vw4cLoNpfLpaKiIlVVVZ30OuFwWOFwOPpzJBLR4cOHlZ2dLcdx4r5mAED3MsaoublZeXl5crm++UW9Ho/UwYMH1dHRodzc3Jjtubm5ev/99096nbKyMj3wwAM9sTwAQA+qr6/XkCFDvnF/j0fqVCxcuFALFiyI/hwMBlVQUKD6+np5PJ4ErgwAcCpCoZDy8/M1YMCAbx3X45EaNGiQkpKS1NDQELO9oaFBPp/vpNdxu91yu91f2+7xeIgUAPRi/+gtmx4/uy81NVXjxo1TRUVFdFskElFFRYX8fn9PLwcAYLGEvNy3YMECzZw5U+PHj9fFF1+sRx55REePHtUNN9yQiOUAACyVkEhde+21OnDggBYtWqRAIKALL7xQr7zyytdOpgAAnNkS8jmp0xUKheT1ehUMBnlPCgB6oc4+jvPdfQAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCs1eVIbdq0SVdccYXy8vLkOI6ee+65mP3GGC1atEhnnXWW+vXrp6KiIn344YcxYw4fPqwZM2bI4/EoMzNTs2bN0pEjR07rjgAA+p4uR+ro0aMaO3asli1bdtL9S5Ys0dKlS/X4449ry5Yt6t+/v4qLi9XS0hIdM2PGDO3atUvl5eVat26dNm3apJtvvvnU7wUAoG8yp0GSWbt2bfTnSCRifD6feeihh6LbmpqajNvtNk8//bQxxpj33nvPSDLbtm2Ljlm/fr1xHMd89tlnnZo3GAwaSSYYDJ7O8gEACdLZx/FufU9qz549CgQCKioqim7zer2aOHGiqqqqJElVVVXKzMzU+PHjo2OKiorkcrm0ZcuWk95uOBxWKBSKuQAA+r5ujVQgEJAk5ebmxmzPzc2N7gsEAsrJyYnZn5ycrKysrOiYryorK5PX641e8vPzu3PZAABL9Yqz+xYuXKhgMBi91NfXJ3pJAIAe0K2R8vl8kqSGhoaY7Q0NDdF9Pp9PjY2NMfvb29t1+PDh6Jivcrvd8ng8MRcAQN/XrZEaPny4fD6fKioqottCoZC2bNkiv98vSfL7/WpqalJ1dXV0zIYNGxSJRDRx4sTuXA4AoJdL7uoVjhw5or///e/Rn/fs2aOdO3cqKytLBQUFmj9/vn7zm99oxIgRGj58uO69917l5eXpyiuvlCSNGjVKl156qW666SY9/vjjamtr05w5c/Szn/1MeXl53XbHAAB9QFdPG3z99deNpK9dZs6caYz54jT0e++91+Tm5hq3222mTJliamtrY27j0KFDZvr06SYjI8N4PB5zww03mObm5m4/dREAYKfOPo47xhiTwEaeklAoJK/Xq2AwyPtTANALdfZxvFec3QcAODMRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArNWlSJWVlWnChAkaMGCAcnJydOWVV6q2tjZmTEtLi0pLS5Wdna2MjAxNmzZNDQ0NMWPq6upUUlKi9PR05eTk6I477lB7e/vp3xsAQJ/SpUhVVlaqtLRUmzdvVnl5udra2jR16lQdPXo0Oua2227Tiy++qDVr1qiyslL79u3T1VdfHd3f0dGhkpIStba26q233tITTzyhVatWadGiRd13rwAAfYM5DY2NjUaSqaysNMYY09TUZFJSUsyaNWuiY3bv3m0kmaqqKmOMMS+//LJxuVwmEAhEx6xYscJ4PB4TDoc7NW8wGDSSTDAYPJ3lAwASpLOP46f1nlQwGJQkZWVlSZKqq6vV1tamoqKi6JiRI0eqoKBAVVVVkqSqqiqNHj1aubm50THFxcUKhULatWvXSecJh8MKhUIxFwBA33fKkYpEIpo/f74mT56sCy64QJIUCASUmpqqzMzMmLG5ubkKBALRMV8O1In9J/adTFlZmbxeb/SSn59/qssGAPQipxyp0tJSvfvuu1q9enV3ruekFi5cqGAwGL3U19fHfU4AQOIln8qV5syZo3Xr1mnTpk0aMmRIdLvP51Nra6uamppink01NDTI5/NFx2zdujXm9k6c/XdizFe53W653e5TWSoAoBfr0jMpY4zmzJmjtWvXasOGDRo+fHjM/nHjxiklJUUVFRXRbbW1taqrq5Pf75ck+f1+1dTUqLGxMTqmvLxcHo9HhYWFp3NfAAB9TJeeSZWWluqpp57S888/rwEDBkTfQ/J6verXr5+8Xq9mzZqlBQsWKCsrSx6PR3PnzpXf79ekSZMkSVOnTlVhYaGuu+46LVmyRIFAQPfcc49KS0t5tgQAiOEYY0ynBzvOSbevXLlSP//5zyV98WHe22+/XU8//bTC4bCKi4u1fPnymJfy9u7dq9mzZ2vjxo3q37+/Zs6cqcWLFys5uXPNDIVC8nq9CgaD8ng8nV0+AMASnX0c71KkbEGkAKB36+zjON/dBwCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArJWc6AUA+D/GmG/c5zhOD64EsAORAixgOjrU3tys0Ntvq2nbNrXU16vj+HElezzqf955GvjP/6z0c89VUv/+xApnFCIFJFgkHFbT5s1qePFFHfvwQ+lLz6baDhzQ8Y8+0qHXX5f3oouUc+WVyhg1ilDhjEGkgAQyxujAa68psGaN2puavnlca6uaNm9Wy/79GnLjjfJceCGhwhmBEyeABDEdHTr0t79p35NPfmugvqxl717V/+EPCm7fLhOJxHeBgAV4JgUkyNEPPlBgzRpFjh2L2f7Z0aPacfiwmtvaNDgtTf7Bg9U/JSW6P7xvnz574gkl9eunjO98h2dU6NOIFJAAkbY2BbdvVzgQiG4zxmjPkSO6b8cOfXLkiFo6OuRJSdEFAwfq/02YoBTX/73w0VJXp72PPabBl16qnJ/8RI6LF0XQN/EnG0iAtkOH1PDsszHbPj5yRDe9+aZ2B4M63tEhIynY1qY3Gxs1b8sWHWppiRkf3rdPnz7xhALPPivT3t6Dqwd6DpECEsAYI9PREbPtkV27FGxrO+n4rQcPqnzfvq/v6OjQvn//dzW88MLXbg/oC4gU0Afse+op7Xv6aXUcO/atHwgGehsiBfQBprVVgTVrVP+nP6nt0KFELwfoNkQKsERJfr5SvuFMvWEZGRqTlfXtN2CMDv3tb/p01Sq1fPZZHFYI9DwiBSRAiterrB/8IGZbcV6e7vvud5WWlBT9i5nkOMp2u/XbCRNUmJnZqdv+/M039fcHH1RzTQ0v/aHX4xR0IAFc/fppoN+v4Pbt6mhulvTFF8gW5+VpSHq61n36qQ61tGhYRoauHT5c2W5352+8o0Ph/fv14f33a8T992vA6NFxuhdA/DmmF/5TKxQKyev1KhgMyuPxJHo5wCmJhMPav2aNGv7617idmZeSlaX8W25R5oQJcpKS4jIHcCo6+zjOy31AgrjcbuVedZUGXHhh3OZoO3xYe5cu1aENGxT5htPbAZsRKSCBktLTlZSWFtc5Oo4cUd3y5Wp8/nm1/+9Li0BvQaSABHIcRwW33qqBkyfHdR7T0aH9zzyjTx59VK0HD8Z1LqA7ESkgwZIHDNDQuXM1cPJkufr1i9s8kWPHFNy6VR/ce6/aOvmt60CiESnAAknp6Rp+xx3K/8UvlJqTE9e5wp99pr8/8ICO793LKeqwHpECLOG4XMr6/veVf9NNSh08OK5zHfvoI+15+GEd/+QTQgWrESnAIq6UFHknTNCI++9Xam5uXOc6/vHH+ug3v1HzO+9w5h+sRaQAyzgul9Ly8zXi/vuVfu65cZ2r9cAB7f3973V440ZCBSsRKcBSaWefrWELFsidlxfXeVobG/XZf/yH6las4FfSwzpECrBYv/x8jXzoIeVedVVcz/xrb2rSoYoKffLII2o/ciRu8wBdRaQAyyUPGKCzrr1WeTNmyJWeHr+JjNHhjRtV/8c/qj0Uit88QBcQKaAXSEpP1+BLL9XQOXOkOH8H3+GNG7X3scfUevAgZ/4h4fgWdKCXcKWmauDkyUpKS9MnS5eqPY4fyG3avFntzc06++c/V/9/+ic53/B7roB445kU0Is4jiPv+PEaOmeOkuL50p+kI7t2qW7FCh185ZW4zgN8GyIF9EKZF1+s8xYtkmf8eMkVv7/Gxz/+WJ+uXKnGl17izD8kBJECeqn+o0Zp6Jw5GlRcHNffFRVpaVH9H/+oxnXrZNrb4zYPcDJECuilHMdRalaWhsycqdyrrorvZMbo07/8RQ3PP6+O48fjOxfwJUQK6OVc/frprGuv1VnTp8f3t+9GItr39NPa9+STagsGOfMPPYKz+4BeznEcOW63zvrZz+QkJ+vga6+ptaEhLnOZ1lY1vvCC2j7/XGdff73ccf5+QYBnUkAf4TiOfNdco/ybb1a/YcPiOtfnb76pvUuXqvmdd+I6D0CkgD7EcRx5L7pIQ3/5S/UfOTJ+E0Uiaq6p0cdLluhobS0v/SFuiBTQxzhJSUo/91yde9dd8owbF9e52kMhfbBokZpramQ6OuI6F85MRArogxzHUbLXq6Fz58o7cWJc54ocP65PHnlEh994g1PU0e2IFNBHnThFffhtt8k7YYJcaWlxm6vt4EHVLVumA6++qkg4HLd5cOYhUkAfl5SernPvukt5112nlOzsuM0TaWnRpytX6tNVq9Syf3/c5sGZhUgBZwAnKUmDp07VkFmzlJyZGbd5TGurDrz0kvb89rf8Xip0CyIFnCFcbrcGfu97Om/RIqUOHhzXuY598IE+uPtutR46xJl/OC1ECjiDOC7XF2f+3XNP3D9LdXzPHn28eLGOffQRocIpI1LAGcZxHKUPH67ht9+ulEGD4jrX0dpaffLIIzr24YeECqekS5FasWKFxowZI4/HI4/HI7/fr/Xr10f3t7S0qLS0VNnZ2crIyNC0adPU8JWvZ6mrq1NJSYnS09OVk5OjO+64Q+2ctgr0uH5Dh2rUww9r8OWXx/XMv5a6On38b/+mprfeUqS1NW7zoG/qUqSGDBmixYsXq7q6Wtu3b9cPf/hD/fSnP9WuXbskSbfddptefPFFrVmzRpWVldq3b5+uvvrq6PU7OjpUUlKi1tZWvfXWW3riiSe0atUqLVq0qHvvFYBOSfF6lTdjhnz/+q9yUlPjNk/rgQOq//Ofte/JJ/ksFbrEMaf5HDwrK0sPPfSQrrnmGg0ePFhPPfWUrrnmGknS+++/r1GjRqmqqkqTJk3S+vXr9eMf/1j79u1T7v9+MeXjjz+uX//61zpw4IBSO/mXJBQKyev1KhgMyuPxnM7yAUiKhMM6vGmT6pYvj+83RziOBhUXq+Dmm+Uk8/3WZ7LOPo6f8ntSHR0dWr16tY4ePSq/36/q6mq1tbWpqKgoOmbkyJEqKChQVVWVJKmqqkqjR4+OBkqSiouLFQqFos/GTiYcDisUCsVcAHQfJzVV2VOmaPivfqXkeP7DzxgdfPVV7V2xgl/3gU7pcqRqamqUkZEht9utW265RWvXrlVhYaECgYBSU1OV+ZXPYOTm5ioQCEiSAoFATKBO7D+x75uUlZXJ6/VGL/n5+V1dNoBv4TiOHJdLA/1+Fdx6a1xf+pMxOlRervo//lFtn38ev3nQJ3Q5Uueff7527typLVu2aPbs2Zo5c6bee++9eKwtauHChQoGg9FLfX19XOcDzmSZfr9G3H+/BowZIzlO3Ob5/I03VPfYYzpeX88zKnyjLkcqNTVV5513nsaNG6eysjKNHTtWjz76qHw+n1pbW9XU1BQzvqGhQT6fT5Lk8/m+drbfiZ9PjDkZt9sdPaPwxAVAfDiOo4zvfEdD581T9g9+ILni9EkVYxTcvl2fPPKIPn/zTUKFkzrtP32RSEThcFjjxo1TSkqKKioqovtqa2tVV1cnv98vSfL7/aqpqVFjY2N0THl5uTwejwoLC093KQC6ieM4cg8erCE33qjBl10W17mOffihPnn4YR0qL4/rPOidunR6zcKFC3XZZZepoKBAzc3Neuqpp7Rx40a9+uqr8nq9mjVrlhYsWKCsrCx5PB7NnTtXfr9fkyZNkiRNnTpVhYWFuu6667RkyRIFAgHdc889Ki0tldvtjssdBHDqkgYM0NnXXy9XWpoan3submf+mbY21f/5zzLGaFBRkeRyyYnjS43oPboUqcbGRl1//fXav3+/vF6vxowZo1dffVU/+tGPJEkPP/ywXC6Xpk2bpnA4rOLiYi1fvjx6/aSkJK1bt06zZ8+W3+9X//79NXPmTD344IPde68AdAvHcZTUr5/Ovv56OcnJCvzXf0lxelkucvy46pYtUyQc1uDiYjn8wxXqhs9JJQKfkwJ6nunoUOOLL+rga6+p5dNP4zaPk5Ii3zXXKOeKK5SckRG3eZBYcf+cFIAzi5OUpJwrrtDQX/5S6eeeG7d5TFubAs88o09XruQUdRApAJ3nJCWp//nn65w771TG6NFxm8e0telQebn2LlumI7t3x20e2I9IAegSx3GUmpOjYfPmyXPRRXGdK7h1qz555BE1bdkS369rgrWIFIAucxxH7pwcnfOrX2nA2LFyxfEkh/D+/drzu98p8Ne/quP48bjNAzsRKQCnLCk9XectWiTftdfG9dfSR44fV2DNGgWefVbtR4/GbR7Yh0gBOC2ulBTllJRoyA03KKl//7jNEwmHdeCllxTavl0mEonbPLALkQJw2pL69VPWJZfo3Lvvjutv++04ckR7ly9Xe3Nz3OaAXYgUgG7huFzK+M53dM6vf620IUPiNk+kpUUHXnklbrcPuxApAN3GcRxlnH++zvnVr5Ts9cbnW9SNUXDbtu6/XViJSAHodmlDh6pw6VJlFxXF9cw/9H1ECkC3cxxHKQMH6uzrrlPulVfKSUpK9JLQSxEpAHGTkpmp3Kuv1tk33CAnuUvfZw1IIlIA4syVlqacH/9YQ3/5SyV10xfGpp19drfcDuxHpADEleM4clwuZX//+yq45ZbTf+nP5VLe9OndszhYj+ffAHrMwH/5FyV7PNq/evUXXxzb1d8U5HLJd801Sh08OD4LhHV4JgWgxziOowFjx2ro/PkaOHly166bnKys739fgy+/XHLx0HWm4P80gB7lOI7SfD4N+cUvlD1lSqevlzl5svKmT1dqVha/Wv4Mwst9ABIiZeBA5d90kwb96EdqeP55Nb/zjiKtrTLt7V+8DOhyyZWSopTsbA2+9FJl//CHSuY3cZ9xiBSAhHAcR0np6eo/apTOOf98Hf3gAx3ZvVvhQECmrU1JAwYo/Zxz5Bk7Nq7fsA67ESkACeU4jpSUpIxRo5QxalSilwPL8J4UAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCw1mlFavHixXIcR/Pnz49ua2lpUWlpqbKzs5WRkaFp06apoaEh5np1dXUqKSlRenq6cnJydMcdd6i9vf10lgIA6INOOVLbtm3TH/7wB40ZMyZm+2233aYXX3xRa9asUWVlpfbt26err746ur+jo0MlJSVqbW3VW2+9pSeeeEKrVq3SokWLTv1eAAD6JnMKmpubzYgRI0x5ebm55JJLzLx584wxxjQ1NZmUlBSzZs2a6Njdu3cbSaaqqsoYY8zLL79sXC6XCQQC0TErVqwwHo/HhMPhTs0fDAaNJBMMBk9l+QCABOvs4/gpPZMqLS1VSUmJioqKYrZXV1erra0tZvvIkSNVUFCgqqoqSVJVVZVGjx6t3Nzc6Jji4mKFQiHt2rXrpPOFw2GFQqGYCwCg70vu6hVWr16tt99+W9u2bfvavkAgoNTUVGVmZsZsz83NVSAQiI75cqBO7D+x72TKysr0wAMPdHWpAIBerkvPpOrr6zVv3jw9+eSTSktLi9eavmbhwoUKBoPRS319fY/NDQBInC5Fqrq6Wo2NjbrooouUnJys5ORkVVZWaunSpUpOTlZubq5aW1vV1NQUc72Ghgb5fD5Jks/n+9rZfid+PjHmq9xutzweT8wFAND3dSlSU6ZMUU1NjXbu3Bm9jB8/XjNmzIj+d0pKiioqKqLXqa2tVV1dnfx+vyTJ7/erpqZGjY2N0THl5eXyeDwqLCzsprsFAOgLuvSe1IABA3TBBRfEbOvfv7+ys7Oj22fNmqUFCxYoKytLHo9Hc+fOld/v16RJkyRJU6dOVWFhoa677jotWbJEgUBA99xzj0pLS+V2u7vpbgEA+oIunzjxjzz88MNyuVyaNm2awuGwiouLtXz58uj+pKQkrVu3TrNnz5bf71f//v01c+ZMPfjgg929FABAL+cYY0yiF9FVoVBIXq9XwWCQ96cAoBfq7OM4390HALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALBWcqIXcCqMMZKkUCiU4JUAAE7FicfvE4/n36RXRurQoUOSpPz8/ASvBABwOpqbm+X1er9xf6+MVFZWliSprq7uW+/cmS4UCik/P1/19fXyeDyJXo61OE6dw3HqHI5T5xhj1NzcrLy8vG8d1ysj5XJ98Vaa1+vlD0EneDwejlMncJw6h+PUORynf6wzTzI4cQIAYC0iBQCwVq+MlNvt1n333Se3253opViN49Q5HKfO4Th1DsepeznmH53/BwBAgvTKZ1IAgDMDkQIAWItIAQCsRaQAANbqlZFatmyZhg0bprS0NE2cOFFbt25N9JJ61KZNm3TFFVcoLy9PjuPoueeei9lvjNGiRYt01llnqV+/fioqKtKHH34YM+bw4cOaMWOGPB6PMjMzNWvWLB05cqQH70V8lZWVacKECRowYIBycnJ05ZVXqra2NmZMS0uLSktLlZ2drYyMDE2bNk0NDQ0xY+rq6lRSUqL09HTl5OTojjvuUHt7e0/elbhasWKFxowZE/3gqd/v1/r166P7OUYnt3jxYjmOo/nz50e3cazixPQyq1evNqmpqeYvf/mL2bVrl7nppptMZmamaWhoSPTSeszLL79s7r77bvPss88aSWbt2rUx+xcvXmy8Xq957rnnzP/8z/+Yn/zkJ2b48OHm+PHj0TGXXnqpGTt2rNm8ebP57//+b3PeeeeZ6dOn9/A9iZ/i4mKzcuVK8+6775qdO3eayy+/3BQUFJgjR45Ex9xyyy0mPz/fVFRUmO3bt5tJkyaZ733ve9H97e3t5oILLjBFRUVmx44d5uWXXzaDBg0yCxcuTMRdiosXXnjBvPTSS+aDDz4wtbW15q677jIpKSnm3XffNcZwjE5m69atZtiwYWbMmDFm3rx50e0cq/jodZG6+OKLTWlpafTnjo4Ok5eXZ8rKyhK4qsT5aqQikYjx+XzmoYceim5ramoybrfbPP3008YYY9577z0jyWzbti06Zv369cZxHPPZZ5/12Np7UmNjo5FkKisrjTFfHJOUlBSzZs2a6Jjdu3cbSaaqqsoY88U/BlwulwkEAtExK1asMB6Px4TD4Z69Az1o4MCB5k9/+hPH6CSam5vNiBEjTHl5ubnkkkuikeJYxU+vermvtbVV1dXVKioqim5zuVwqKipSVVVVAldmjz179igQCMQcI6/Xq4kTJ0aPUVVVlTIzMzV+/PjomKKiIrlcLm3ZsqXH19wTgsGgpP/7cuLq6mq1tbXFHKeRI0eqoKAg5jiNHj1aubm50THFxcUKhULatWtXD66+Z3R0dGj16tU6evSo/H4/x+gkSktLVVJSEnNMJP48xVOv+oLZgwcPqqOjI+Z/siTl5ubq/fffT9Cq7BIIBCTppMfoxL5AIKCcnJyY/cnJycrKyoqO6UsikYjmz5+vyZMn64ILLpD0xTFITU1VZmZmzNivHqeTHccT+/qKmpoa+f1+tbS0KCMjQ2vXrlVhYaF27tzJMfqS1atX6+2339a2bdu+to8/T/HTqyIFnIrS0lK9++67euONNxK9FCudf/752rlzp4LBoJ555hnNnDlTlZWViV6WVerr6zVv3jyVl5crLS0t0cs5o/Sql/sGDRqkpKSkr50x09DQIJ/Pl6BV2eXEcfi2Y+Tz+dTY2Bizv729XYcPH+5zx3HOnDlat26dXn/9dQ0ZMiS63efzqbW1VU1NTTHjv3qcTnYcT+zrK1JTU3Xeeedp3LhxKisr09ixY/Xoo49yjL6kurpajY2Nuuiii5ScnKzk5GRVVlZq6dKlSk5OVm5uLscqTnpVpFJTUzVu3DhVVFREt0UiEVVUVMjv9ydwZfYYPny4fD5fzDEKhULasmVL9Bj5/X41NTWpuro6OmbDhg2KRCKaOHFij685HowxmjNnjtauXasNGzZo+PDhMfvHjRunlJSUmONUW1ururq6mONUU1MTE/Ty8nJ5PB4VFhb2zB1JgEgkonA4zDH6kilTpqimpkY7d+6MXsaPH68ZM2ZE/5tjFSeJPnOjq1avXm3cbrdZtWqVee+998zNN99sMjMzY86Y6euam5vNjh07zI4dO4wk87vf/c7s2LHD7N271xjzxSnomZmZ5vnnnzfvvPOO+elPf3rSU9C/+93vmi1btpg33njDjBgxok+dgj579mzj9XrNxo0bzf79+6OXY8eORcfccsstpqCgwGzYsMFs377d+P1+4/f7o/tPnDI8depUs3PnTvPKK6+YwYMH96lThu+8805TWVlp9uzZY9555x1z5513GsdxzGuvvWaM4Rh9my+f3WcMxypeel2kjDHm97//vSkoKDCpqanm4osvNps3b070knrU66+/biR97TJz5kxjzBenod97770mNzfXuN1uM2XKFFNbWxtzG4cOHTLTp083GRkZxuPxmBtuuME0Nzcn4N7Ex8mOjySzcuXK6Jjjx4+bW2+91QwcONCkp6ebq666yuzfvz/mdj755BNz2WWXmX79+plBgwaZ22+/3bS1tfXwvYmfG2+80QwdOtSkpqaawYMHmylTpkQDZQzH6Nt8NVIcq/jgV3UAAKzVq96TAgCcWYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCw1v8HNby0aS+GvzEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_18402/3166162692.py:92: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  states = torch.FloatTensor(states).reshape(-1, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.04278993606567383, -1636.4170616846322)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import random\n",
    "from IPython import display\n",
    "\n",
    "\n",
    "#定义模型\n",
    "class Model(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc_statu = torch.nn.Sequential(\n",
    "            torch.nn.Linear(3, 128),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        self.fc_mu = torch.nn.Sequential(\n",
    "            torch.nn.Linear(128, 1),\n",
    "            torch.nn.Tanh(),\n",
    "        )\n",
    "\n",
    "        self.fc_std = torch.nn.Sequential(\n",
    "            torch.nn.Linear(128, 1),\n",
    "            torch.nn.Softplus(),\n",
    "        )\n",
    "\n",
    "    def forward(self, state):\n",
    "        state = self.fc_statu(state)\n",
    "\n",
    "        mu = self.fc_mu(state) * 2.0\n",
    "        std = self.fc_std(state)\n",
    "\n",
    "        return mu, std\n",
    "\n",
    "\n",
    "class PPO:\n",
    "\n",
    "    def __init__(self):\n",
    "        #定义模型\n",
    "        self.model = Model()\n",
    "\n",
    "        self.model_td = torch.nn.Sequential(\n",
    "            torch.nn.Linear(3, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 1),\n",
    "        )\n",
    "\n",
    "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=1e-4)\n",
    "        self.optimizer_td = torch.optim.Adam(self.model_td.parameters(),\n",
    "                                             lr=5e-3)\n",
    "        self.loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    def get_action(self, state):\n",
    "        state = torch.FloatTensor(state).reshape(1, 3)\n",
    "        mu, std = self.model(state)\n",
    "\n",
    "        #根据概率选择一个动作\n",
    "        #action = random.normalvariate(mu=mu.item(), sigma=std.item())\n",
    "        action = torch.distributions.Normal(mu, std).sample().item()\n",
    "\n",
    "        return action\n",
    "\n",
    "    def get_data(self):\n",
    "        states = []\n",
    "        rewards = []\n",
    "        actions = []\n",
    "        next_states = []\n",
    "        overs = []\n",
    "\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action = self.get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step([action])\n",
    "\n",
    "            #记录数据样本\n",
    "            states.append(state)\n",
    "            rewards.append(reward)\n",
    "            actions.append(action)\n",
    "            next_states.append(next_state)\n",
    "            overs.append(over)\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "        #[b, 3]\n",
    "        states = torch.FloatTensor(states).reshape(-1, 3)\n",
    "        #[b, 1]\n",
    "        rewards = torch.FloatTensor(rewards).reshape(-1, 1)\n",
    "        #[b, 1]\n",
    "        actions = torch.FloatTensor(actions).reshape(-1, 1)\n",
    "        #[b, 3]\n",
    "        next_states = torch.FloatTensor(next_states).reshape(-1, 3)\n",
    "        #[b, 1]\n",
    "        overs = torch.LongTensor(overs).reshape(-1, 1)\n",
    "\n",
    "        return states, rewards, actions, next_states, overs\n",
    "\n",
    "    def test(self, play):\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #记录反馈值的和,这个值越大越好\n",
    "        reward_sum = 0\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action = self.get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            state, reward, over, _ = env.step([action])\n",
    "            reward_sum += reward\n",
    "\n",
    "            #打印动画\n",
    "            if play and random.random() < 0.2:  #跳帧\n",
    "                display.clear_output(wait=True)\n",
    "                show()\n",
    "\n",
    "        return reward_sum\n",
    "\n",
    "    #优势函数\n",
    "    def get_advantages(self, deltas):\n",
    "        advantages = []\n",
    "\n",
    "        #反向遍历deltas\n",
    "        s = 0.0\n",
    "        for delta in deltas[::-1]:\n",
    "            s = 0.9 * 0.9 * s + delta\n",
    "            advantages.append(s)\n",
    "\n",
    "        #逆序\n",
    "        advantages.reverse()\n",
    "        return advantages\n",
    "    \n",
    "    def train(self, states, rewards, actions, next_states, overs):\n",
    "        #偏移reward,便于训练\n",
    "        rewards = (rewards + 8) / 8\n",
    "\n",
    "        #计算values和targets\n",
    "        #[b, 3] -> [b, 1]\n",
    "        values = self.model_td(states)\n",
    "\n",
    "        #[b, 3] -> [b, 1]\n",
    "        targets = self.model_td(next_states).detach()\n",
    "        targets = targets * 0.98\n",
    "        targets *= (1 - overs)\n",
    "        targets += rewards\n",
    "\n",
    "        #计算优势,这里的advantages有点像是策略梯度里的reward_sum\n",
    "        #只是这里计算的不是reward,而是target和value的差\n",
    "        #[b, 1]\n",
    "        deltas = (targets - values).squeeze(dim=1).tolist()\n",
    "        advantages = self.get_advantages(deltas)\n",
    "        advantages = torch.FloatTensor(advantages).reshape(-1, 1)\n",
    "\n",
    "        #取出每一步动作的概率\n",
    "        #[b, 3] -> [b, 1],[b, 1]\n",
    "        mu, std = self.model(states)\n",
    "        #[b, 1]\n",
    "        old_probs = torch.distributions.Normal(mu, std)\n",
    "        old_probs = old_probs.log_prob(actions).exp().detach()\n",
    "\n",
    "        #每批数据反复训练10次\n",
    "        for _ in range(10):\n",
    "            #重新计算每一步动作的概率\n",
    "            #[b, 3] -> [b, 1],[b, 1]\n",
    "            mu, std = self.model(states)\n",
    "            #[b, 1]\n",
    "            new_probs = torch.distributions.Normal(mu, std)\n",
    "            new_probs = new_probs.log_prob(actions).exp()\n",
    "\n",
    "            #求出概率的变化\n",
    "            #[b, 1] - [b, 1] -> [b, 1]\n",
    "            ratios = new_probs / old_probs\n",
    "\n",
    "            #计算截断的和不截断的两份loss,取其中小的\n",
    "            #[b, 1] * [b, 1] -> [b, 1]\n",
    "            surr1 = ratios * advantages\n",
    "            #[b, 1] * [b, 1] -> [b, 1]\n",
    "            surr2 = torch.clamp(ratios, 0.8, 1.2) * advantages\n",
    "\n",
    "            loss = -torch.min(surr1, surr2)\n",
    "            loss = loss.mean()\n",
    "\n",
    "            #重新计算value,并计算时序差分loss\n",
    "            values = self.model_td(states)\n",
    "            loss_td = self.loss_fn(values, targets)\n",
    "\n",
    "            #更新参数\n",
    "            self.optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            self.optimizer.step()\n",
    "\n",
    "            self.optimizer_td.zero_grad()\n",
    "            loss_td.backward()\n",
    "            self.optimizer_td.step()\n",
    "\n",
    "\n",
    "teacher = PPO()\n",
    "\n",
    "teacher.train(*teacher.get_data())\n",
    "\n",
    "teacher.get_action([1, 2, 3]), teacher.test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 8251,
     "status": "ok",
     "timestamp": 1650011468229,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "BQXVYW2T_DcQ",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -1487.7050151939607\n",
      "500 -964.3770685219636\n",
      "1000 -696.0384044881057\n",
      "1500 -185.08530909170008\n",
      "2000 -366.2742842420944\n",
      "2500 -321.5826157837225\n",
      "3000 -184.39423223403782\n",
      "3500 -312.4312647284078\n",
      "4000 -145.82879414707298\n",
      "4500 -328.7718385283696\n",
      "5000 -167.0804918331225\n",
      "5500 -388.6714654224169\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-267.37763297259966"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(6000):\n",
    "    teacher.train(*teacher.get_data())\n",
    "\n",
    "    if i % 500 == 0:\n",
    "        test_result = sum([teacher.test(play=False) for _ in range(10)]) / 10\n",
    "        print(i, test_result)\n",
    "\n",
    "teacher.test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.PPO at 0x7f3b9b4c97c0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#初始化学生模型\n",
    "student = PPO()\n",
    "\n",
    "student"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.5350],\n",
       "        [0.5032]], grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#定义鉴别器网络,它的任务是鉴定一批数据是出自teacher还是student\n",
    "class Discriminator(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.sequential = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 1),\n",
    "            torch.nn.Sigmoid(),\n",
    "        )\n",
    "\n",
    "    def forward(self, states, actions):\n",
    "        cat = torch.cat([states, actions], dim=1)\n",
    "\n",
    "        return self.sequential(cat)\n",
    "\n",
    "\n",
    "discriminator = Discriminator()\n",
    "\n",
    "discriminator(torch.randn(2, 3), torch.randn(2, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -1449.3359284945432\n",
      "500 -1472.0271150654457\n",
      "1000 -1104.0216695477507\n",
      "1500 -999.4593262113846\n",
      "2000 -791.0525053569535\n",
      "2500 -760.0513923951504\n",
      "3000 -224.85173278083903\n",
      "3500 -437.52168137990657\n",
      "4000 -557.6787322030593\n",
      "4500 -212.0491915372591\n",
      "5000 -240.51252221468502\n",
      "5500 -491.65894908048574\n",
      "6000 -378.73559085634986\n",
      "6500 -283.29922912079127\n",
      "7000 -1098.4314353987884\n",
      "7500 -394.1215611282811\n",
      "8000 -211.80704640581703\n",
      "8500 -228.5568271585168\n",
      "9000 -239.67402237864704\n",
      "9500 -554.5628311917744\n",
      "10000 -715.9393652455734\n",
      "10500 -730.4719044011002\n",
      "11000 -544.4484119269265\n",
      "11500 -490.1208476937456\n",
      "12000 -701.9949409653982\n",
      "12500 -646.4799634249957\n",
      "13000 -411.49168273678305\n",
      "13500 -220.82545963440066\n",
      "14000 -161.89409938640551\n",
      "14500 -110.28688864587521\n",
      "15000 -224.29416517987096\n",
      "15500 -192.85583186181094\n",
      "16000 -647.6253536817621\n",
      "16500 -291.2458463068299\n",
      "17000 -231.40140874677567\n",
      "17500 -295.4659108896758\n",
      "18000 -267.3246686284553\n",
      "18500 -228.6499147669064\n",
      "19000 -297.84286192756116\n",
      "19500 -362.117464314271\n"
     ]
    }
   ],
   "source": [
    "#模仿学习\n",
    "def copy_learn():\n",
    "    optimizer = torch.optim.Adam(discriminator.parameters(), lr=1e-4)\n",
    "    bce_loss = torch.nn.BCELoss()\n",
    "\n",
    "    for i in range(20000):\n",
    "        #使用训练好的模型获取一批教师数据\n",
    "        with torch.no_grad():\n",
    "            teacher_states, _, teacher_actions, _, _ = teacher.get_data()\n",
    "\n",
    "        #使用学生模型获取一局游戏的数据,不需要reward\n",
    "        states, _, actions, next_states, overs = student.get_data()\n",
    "\n",
    "        #使用鉴别器鉴定两批数据是来自教师的还是学生的\n",
    "        prob_teacher = discriminator(teacher_states, teacher_actions)\n",
    "        prob_student = discriminator(states, actions)\n",
    "\n",
    "        #老师的用0表示,学生的用1表示,计算二分类loss\n",
    "        loss_teacher = bce_loss(prob_teacher, torch.zeros_like(prob_teacher))\n",
    "        loss_student = bce_loss(prob_student, torch.ones_like(prob_student))\n",
    "        loss = loss_teacher + loss_student\n",
    "\n",
    "        #调整鉴别器的loss\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        #使用一批数据来自学生的概率作为reward,取log,再符号取反\n",
    "        #因为鉴别器会把学生数据的概率贴近1,所以目标是让鉴别器无法分辨,这是一种对抗网络的思路\n",
    "        rewards = -prob_student.log().detach()\n",
    "\n",
    "        #消除模型中的reward偏移\n",
    "        rewards = rewards * 8 - 8\n",
    "\n",
    "        #更新学生模型参数,使用PPO模型本身的更新方式\n",
    "        student.train(states, rewards, actions, next_states, overs)\n",
    "\n",
    "        if i % 500 == 0:\n",
    "            test_result = sum([student.test(play=False)\n",
    "                               for _ in range(10)]) / 10\n",
    "\n",
    "            print(i, test_result)\n",
    "\n",
    "\n",
    "copy_learn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAm50lEQVR4nO3df3SU9YHv8c/8SCY/Z5KAmZglAVypNOVHLSBM3du6JSVS/EHFPa6H47LK0SsGj8gez8JWcettDx57u1q3ij3bs+reo+LBij9Y0c0NNLZL5EeEGhDRWpQgTCI/MpOEZDKZ+d4/lLmOgk3IZOY78f06Z85pnueZb77P0zhvZuaZeRzGGCMAACzkzPQEAAA4GyIFALAWkQIAWItIAQCsRaQAANYiUgAAaxEpAIC1iBQAwFpECgBgLSIFALBWxiL1yCOPaMKECcrLy9Ps2bO1Y8eOTE0FAGCpjETq2Wef1cqVK3XvvffqzTff1PTp01VXV6eOjo5MTAcAYClHJr5gdvbs2Zo1a5Z++ctfSpLi8biqqqp0++23a9WqVemeDgDAUu50/8L+/n61tLRo9erViWVOp1O1tbVqbm4+430ikYgikUji53g8rhMnTmjMmDFyOBwjPmcAQGoZY9TV1aXKyko5nWd/US/tkTp27JhisZj8fn/Scr/fr3feeeeM91m7dq1+/OMfp2N6AIA0amtr07hx4866Pu2ROherV6/WypUrEz+HQiFVV1erra1NXq83gzMDAJyLcDisqqoqFRcXf+l2aY/U2LFj5XK51N7enrS8vb1dFRUVZ7yPx+ORx+P5wnKv10ukACCL/bm3bNJ+dl9ubq5mzJihxsbGxLJ4PK7GxkYFAoF0TwcAYLGMvNy3cuVKLVmyRDNnztQll1yihx56SD09PbrxxhszMR0AgKUyEqnrrrtOH3/8sdasWaNgMKhvfvObevXVV79wMgUA4KstI5+TGq5wOCyfz6dQKMR7UgCQhQb7OM539wEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCw1pAj9frrr+vKK69UZWWlHA6HXnjhhaT1xhitWbNG559/vvLz81VbW6v33nsvaZsTJ05o8eLF8nq9Kikp0dKlS9Xd3T2sHQEAjD5DjlRPT4+mT5+uRx555IzrH3jgAT388MN67LHHtH37dhUWFqqurk59fX2JbRYvXqx9+/apoaFBmzZt0uuvv65bbrnl3PcCADA6mWGQZDZu3Jj4OR6Pm4qKCvOzn/0ssayzs9N4PB7zzDPPGGOMefvtt40ks3PnzsQ2mzdvNg6Hw3z00UeD+r2hUMhIMqFQaDjTBwBkyGAfx1P6ntTBgwcVDAZVW1ubWObz+TR79mw1NzdLkpqbm1VSUqKZM2cmtqmtrZXT6dT27dvPOG4kElE4HE66AQBGv5RGKhgMSpL8fn/Scr/fn1gXDAZVXl6etN7tdqusrCyxzeetXbtWPp8vcauqqkrltAEAlsqKs/tWr16tUCiUuLW1tWV6SgCANEhppCoqKiRJ7e3tScvb29sT6yoqKtTR0ZG0fmBgQCdOnEhs83kej0derzfpBgAY/VIaqYkTJ6qiokKNjY2JZeFwWNu3b1cgEJAkBQIBdXZ2qqWlJbHNli1bFI/HNXv27FROBwCQ5dxDvUN3d7f++Mc/Jn4+ePCg9uzZo7KyMlVXV2vFihX6yU9+okmTJmnixIm65557VFlZqYULF0qSvv71r+vyyy/XzTffrMcee0zRaFTLly/X3/7t36qysjJlOwYAGAWGetrg1q1bjaQv3JYsWWKM+eQ09Hvuucf4/X7j8XjM3LlzzYEDB5LGOH78uLn++utNUVGR8Xq95sYbbzRdXV0pP3URAGCnwT6OO4wxJoONPCfhcFg+n0+hUIj3pwAgCw32cTwrzu4DAHw1ESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkP+FnQAdjHGSPG4TCwmE49LxkgOhxxOpxxutxxO/i2K7EWkgCwWj0YVOXpUnTt2KLxrl3oPH1a8t1euoiLljx+v0kBAxd/8pjzl5XK4XJmeLjBkRArIUvH+fh1raNCx115T74cffvIM6lMDJ0+q6+RJdbe2quDCC1V+1VUq/au/ksPhyOCMgaEjUkAWig8MKLhxo9qff17x3t6zbmdiMfUcOKC2f/s3xfv7Neayy3hGhazCi9VAlolHo/p482a1P/fclwbqswY6O/XRf/yHTjY3y8RiIzxDIHWIFJBlTr33njpefFHxSGRI9xs4eVLBZ59V//HjIzQzIPWIFJBF4tGoOnfuVH9Hxzndv/fDDxXevVtZeEFufEURKSCLxHp61P6b3wxrjI+efDLpJAvAZkQKyCIDoVCmpwCkFZECskj3/v2ZngKQVkQKyCIdL72U6SkAaUWkgCyRspMd+EAvsgiRArJEvLc3JZ9x8s2aRaiQNYgUkCUiwaDi/f3DHsf7zW8OfzJAmhApIEuEWlpScnZfXmVlCmYDpAeRArJE9ORJmYGBYY/jyM3li2aRNYgUkAXiAwMy0eiwx/Gcf75ceXkpmBGQHkQKyAKx7m5FT54c9jiFkyfLVVycghkB6UGkgCwQPXFCkSNHhj2Ox++Xy+NJwYyA9CBSQBboP3ZMfYcPD3scV36+HG4uI4fsQaQAyxljUnLChBwOiQseIssQKcB28XhKnkXljRun4qlTUzAhIH2IFGC5eDSqk9u2DXscV2GhcsrKUjAjIH2IFGC7WEy9H3ww7GGceXlyc2YfsgyRAiwXT8HnoyTJ4XDI4eQ/eWQX/mIBy0WCweFfSdfpVNE3vpGaCQFpRKQAy/W8++6wx3C43fJefHEKZgOkF5ECLHdi69ZhP5NyOBzy8MWyyEJECrBYKi906OSbJpCFiBRgsejJk4pHIsMex1VYKL73HNmISAEW69m/X9EUXEPqvB/8QOLMPmQh/moBi516/33FurqGPU7euHFcMh5ZiUgBljLGpOw9qZzS0pSMA6QbkQIsFe/vV6y7O2XjcTVeZCMiBVgq1tOj/mPHhj1O4eTJfGcfshaRAiwV+egjhd98c9jj5E+YILfXm4IZAelHpABLGWOG/3VIknJ8Pjlzc1MwIyD9iBRgIWOMBlJw6rkkyeXii2WRtfjLBWwUj6fm8hwej3J8vuHPB8gQIgVYyMTjCu3aNexxcv1+FU+fnoIZAZlBpAAbpeiS8a78fOWOHZuCCQGZQaQACw10d6fkpAmHyyVHTk4KZgRkBpECLNTX1payb5vgQ7zIZkQKsFDHyy9LsdjwBnE6NXb+/NRMCMgQIgVYKNbTM+wxHE6nii66KAWzATKHSAGWiff3ywz3WdSn3CUlKRkHyBQiBVim/9gxxXp7UzIW70ch2xEpwDKn/vQnRU+eHPY4+RdcwIUOkfX4CwYsEzl8OCUXOhxz2WVyuFwpmBGQOUQKsIiJx2Xi8ZSM5fmLv+BqvMh6RAqwSLyvT/0ff5ySsRxOJ+9JIesRKcAi/R0dCrW0DHucvOpqLhmPUWFIkVq7dq1mzZql4uJilZeXa+HChTpw4EDSNn19faqvr9eYMWNUVFSkRYsWqb29PWmbQ4cOacGCBSooKFB5ebnuuusuDQwMDH9vgCwXj0Q00Nk57HHyxo3j9HOMCkOKVFNTk+rr6/XGG2+ooaFB0WhU8+bNU89nPnh455136uWXX9aGDRvU1NSkI0eO6Jprrkmsj8ViWrBggfr7+7Vt2zY9+eSTeuKJJ7RmzZrU7RWQhYwxiqfoH2u5ZWVyFRSkZCwgkxxmGF8Q9vHHH6u8vFxNTU36zne+o1AopPPOO09PP/20rr32WknSO++8o69//etqbm7WnDlztHnzZl1xxRU6cuSI/H6/JOmxxx7TP/7jP+rjjz9W7iCuIBoOh+Xz+RQKheTlstgYJYwxOr51qz586KFhj+VftEjjliwZ/qSAETLYx/FhvScV+vTKoWVlZZKklpYWRaNR1dbWJraZPHmyqqur1dzcLElqbm7W1KlTE4GSpLq6OoXDYe3bt++MvycSiSgcDifdgFHHGHX94Q/DHsbhdstdXJyCCQGZd86RisfjWrFihS699FJNmTJFkhQMBpWbm6uSz70W7vf7FQwGE9t8NlCn159edyZr166Vz+dL3Kqqqs512oC94nF1fvqPueHIKS1V8dSpKZgQkHnnHKn6+nrt3btX69evT+V8zmj16tUKhUKJW1tb24j/TiDdUnVpDmdennLPOy8lYwGZ5j6XOy1fvlybNm3S66+/rnHjxiWWV1RUqL+/X52dnUnPptrb21VRUZHYZseOHUnjnT777/Q2n+fxeOTxeM5lqkDW6G9vT82FDnNy5Pb5UjAjIPOG9EzKGKPly5dr48aN2rJliyZOnJi0fsaMGcrJyVFjY2Ni2YEDB3To0CEFAgFJUiAQUGtrqzo6OhLbNDQ0yOv1qqamZjj7AmS13sOHU/ZtE3yIF6PFkJ5J1dfX6+mnn9aLL76o4uLixHtIPp9P+fn58vl8Wrp0qVauXKmysjJ5vV7dfvvtCgQCmjNnjiRp3rx5qqmp0Q033KAHHnhAwWBQd999t+rr63m2hK+0k6+/LhONDm8Qh0Nl3/lOaiYEWGBIkVq3bp0k6bLLLkta/vjjj+vv//7vJUkPPvignE6nFi1apEgkorq6Oj366KOJbV0ulzZt2qRly5YpEAiosLBQS5Ys0X333Te8PQGymDEmJRc6lKSiyZNTMg5gg2F9TipT+JwURpuB7m69/9OfqvssH8MYNKdT0x5/nK9EgvXS8jkpAKkRCQYV/fRzh8PlGMQH4oFsQaQAC3S3tipy+PCwx/FUVsrBhQ4xivDXDGRYKl9xL5k9W46cnJSNB2QakQIyzMRiGujuTslYeePG8UwKowp/zUCGxfv6FPnc5WzOlbu4mEhhVOGvGciwgVBIXW+9lelpAFYiUkCGmWg0JRc6LPrGN5RfXT38CQEWIVJABhljNJCiD/HmjBkjN58bxChDpIAM6/3gg5SM4y4qkjMvLyVjAbYgUkAmGaOjzz6bmrEcDk6awKjDXzSQYSYWG/YYzrw85Y8fn4LZAHYhUkAGxSORlFxDylVUpOJPr5ANjCZECsigvsOHU/NMKjdXueXlKZgRYBciBWRQaOdOxfv7hz+QwyEnXyyLUeicLh8PIDV6P/xQZmBARlI0HtdAPK6YMXI4HCpyuwd9hV0ChdGKSAEZYgYGFIvF9EF3t/aHQvpjOKyP+/rUMzCgstxcrZo2Te5BRqpi0aIRni2QGUQKyJAjH3yg/7Njh3a9956qCgr0jZISjSssVElurvJdLrkGGShJyuObJjBKESkgzeLxuPbu3av/9aMfqfj99/U/L7pI5Xl5ynU6B/3y3ufl+HwpniVgByIFpNHAwIB+97vf6Ze//KXmjx+vKcbI7XCcc5wShnt/wFKc3QekiTFGLS0t+tWvfqX6227TlZdfrpxhPHs6rXjaNE6cwKhFpIA06ejo0KOPPqrrrrtO/+PSSxU9fjwl4xZPn06kMGoRKSANjDF66qmnNH78eM2fP1/q61N3a2tKxs4dO1ZyuVIyFmAbIgWkwQcffKCtW7fq5ptvVl5enmK9vep5992UjO1IwUuGgK2IFDDC4vG4nn/+ec2dO1fjxo2TMUZKwVchSZLT45HT40nJWICNiBQwwk6cOKF9+/bp+9//fmJZpKPjrNt/1NOjTW1teuZPf9L/PXJEPdHoWbctqqlRwde+pmg0qhMnTqR03oANOAUdGGGHDx9WQUGBysvL5XA4ZIw544UOjTE62N2te3fv1gfd3eqLxeTNydGU0lL971mzlHOGa0W5fT65vV4dOnxYL774opYtW6ZcTqLAKMIzKWCEdXR0qKSkRAUFBYllH7/yyhe2+1N3t27+7//W/lBIvbGYjKRQNKr/7ujQHdu363hf3xfu43C75XC59NJLL2nDhg368MMPR3JXgLQjUsAI6+npUUFBQdIzHDMw8IXtHtq3T6GzvLS349gxNRw5csZ1kUhE69ev1969e/XOO+988p4XMEoQKWCExeNx5eXlyfmZl+uqbrll2OO6CgpUMmeOWlpadOLECQ0MDGjbtm2Kfsl7WEC2IVLACHM4HOrv7088w3E4HMqfOHHY4zrz8lR40UXq7+/XQw89pGuuuUYzZ87kmRRGFSIFjLD8/HydOnXqzz7DWVBVpZyzfN5pQlGRppWVJS1zuFxye7267LLL5HA45Pf7tXDhQk6cwKhCpIARVlZWpq6uLvX29iaW5fh8Kvvrv07arq6yUvdefLHyXK7Ef5guh0NjPB79fNYs1ZSUJG3v/8w1pN59912NHz9eLpeLD/ZiVOEUdGCEVVVVKRwO6+TJkyotLZXD4ZAzP1+lgYBCu3Yp1tUl6ZOXAesqKzWuoECbDh/W8b4+TSgq0nUTJ2rM5z6w6zn/fJUGAnI4HBoYGNCOHTt02223ZWL3gBFFpIARVl5ersrKSu3YsUMXXHCBpE+C5L34Yp03f77af/MbmU+/gcLhcGhKaammlJaedTx3aakqb7hBbq9XkvTee++pq6tLU6dOHfmdAdKMl/uAEeZ2u3XFFVfoueee06lTpxLLnR6P/D/8ocq+9z053IP796KrqEjnX3edSi65RA6XS9FoVE8//bSuuOIKFRUVjdQuABlDpIA0uOSSS/SXf/mXevLJJ5NOoHAVFGjcTTfJv2iRcsvLz3p/h8ul/AkTVHXzzTpv/nw5c3MVj8fV3NysgwcPauHChWnYCyD9eLkPSAOHw6GbbrpJ9957r2bNmqWZM2fK8ekVed2FhTr/2mvlnTZNJ7dtU/e+fYoEg4pHInIVFSlv3Dj5Zs6Ub+ZM5VdXJ06MCIVCeuaZZzR//nyVfsnLg0A2I1JAmkyaNEl/8zd/o4ceekg/+clPNGHChERwnB6PiqZMUeHXvqZYb6/MwIBMPC6HyyVnTo6cBQVyfvqSoDFGp06d0s9//nONHTtWV199tVxcTwqjFJEC0sTlcunKK69UZ2en7rnnHq1cuVLTpk2T+9P4OBwOOf7MpTeMMTp8+LAee+wx9fb26p//+Z9VWFiYrl0A0o73pIA08ng8uuGGGzR//nz99Kc/1VNPPaVIJCJjzFm/KeL0OmOM3njjDa1atUo5OTlatWqVvF4vn4vCqOYwWfgdKuFwWD6fT6FQSN5PT8MFsoUxRrFYTHv37tUjjzyi3t5eXXvttZo+fbp8Pp88Ho9cLpfi8bj6+/vV09Oj999/Xxs3btT777+vZcuW6bLLLlNeXh6BQtYa7OM4kQIyKBqNqrGxUa+++qo6OztVXFys/Px8ud1uxWIxRSIRdXd3y+Fw6Nvf/rauuuoqlZWVESdkPSIFZAljjPr6+nT06FEFg0GFw2FFo1G5XC4VFhZq7NixqqqqUnFxMXHCqDHYx3FOnAAyzOFwKD8/XxdccEHiGykAfIITJwAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWIFADAWkQKAGAtIgUAsBaRAgBYi0gBAKxFpAAA1iJSAABrESkAgLWGFKl169Zp2rRp8nq98nq9CgQC2rx5c2J9X1+f6uvrNWbMGBUVFWnRokVqb29PGuPQoUNasGCBCgoKVF5errvuuksDAwOp2RsAwKgypEiNGzdO999/v1paWrRr1y5973vf09VXX619+/ZJku688069/PLL2rBhg5qamnTkyBFdc801ifvHYjEtWLBA/f392rZtm5588kk98cQTWrNmTWr3CgAwOphhKi0tNb/+9a9NZ2enycnJMRs2bEis279/v5FkmpubjTHGvPLKK8bpdJpgMJjYZt26dcbr9ZpIJDLo3xkKhYwkEwqFhjt9AEAGDPZx/Jzfk4rFYlq/fr16enoUCATU0tKiaDSq2traxDaTJ09WdXW1mpubJUnNzc2aOnWq/H5/Ypu6ujqFw+HEs7EziUQiCofDSTcAwOg35Ei1traqqKhIHo9Ht956qzZu3KiamhoFg0Hl5uaqpKQkaXu/369gMChJCgaDSYE6vf70urNZu3atfD5f4lZVVTXUaQMAstCQI3XRRRdpz5492r59u5YtW6YlS5bo7bffHom5JaxevVqhUChxa2trG9HfBwCwg3uod8jNzdWFF14oSZoxY4Z27typX/ziF7ruuuvU39+vzs7OpGdT7e3tqqiokCRVVFRox44dSeOdPvvv9DZn4vF45PF4hjpVAECWG/bnpOLxuCKRiGbMmKGcnBw1NjYm1h04cECHDh1SIBCQJAUCAbW2tqqjoyOxTUNDg7xer2pqaoY7FQDAKDOkZ1KrV6/W/PnzVV1dra6uLj399NP67W9/q9dee00+n09Lly7VypUrVVZWJq/Xq9tvv12BQEBz5syRJM2bN081NTW64YYb9MADDygYDOruu+9WfX09z5QAAF8wpEh1dHTo7/7u73T06FH5fD5NmzZNr732mr7//e9Lkh588EE5nU4tWrRIkUhEdXV1evTRRxP3d7lc2rRpk5YtW6ZAIKDCwkItWbJE9913X2r3CgAwKjiMMSbTkxiqcDgsn8+nUCgkr9eb6ekAAIZosI/jfHcfAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsNK1L333+/HA6HVqxYkVjW19en+vp6jRkzRkVFRVq0aJHa29uT7nfo0CEtWLBABQUFKi8v11133aWBgYHhTAUAMAqdc6R27typX/3qV5o2bVrS8jvvvFMvv/yyNmzYoKamJh05ckTXXHNNYn0sFtOCBQvU39+vbdu26cknn9QTTzyhNWvWnPteAABGJ3MOurq6zKRJk0xDQ4P57ne/a+644w5jjDGdnZ0mJyfHbNiwIbHt/v37jSTT3NxsjDHmlVdeMU6n0wSDwcQ269atM16v10QikUH9/lAoZCSZUCh0LtMHAGTYYB/Hz+mZVH19vRYsWKDa2tqk5S0tLYpGo0nLJ0+erOrqajU3N0uSmpubNXXqVPn9/sQ2dXV1CofD2rdv3xl/XyQSUTgcTroBAEY/91DvsH79er355pvauXPnF9YFg0Hl5uaqpKQkabnf71cwGExs89lAnV5/et2ZrF27Vj/+8Y+HOlUAQJYb0jOptrY23XHHHXrqqaeUl5c3UnP6gtWrVysUCiVubW1tafvdAIDMGVKkWlpa1NHRoW9961tyu91yu91qamrSww8/LLfbLb/fr/7+fnV2dibdr729XRUVFZKkioqKL5ztd/rn09t8nsfjkdfrTboBAEa/IUVq7ty5am1t1Z49exK3mTNnavHixYn/nZOTo8bGxsR9Dhw4oEOHDikQCEiSAoGAWltb1dHRkdimoaFBXq9XNTU1KdotAMBoMKT3pIqLizVlypSkZYWFhRozZkxi+dKlS7Vy5UqVlZXJ6/Xq9ttvVyAQ0Jw5cyRJ8+bNU01NjW644QY98MADCgaDuvvuu1VfXy+Px5Oi3QIAjAZDPnHiz3nwwQfldDq1aNEiRSIR1dXV6dFHH02sd7lc2rRpk5YtW6ZAIKDCwkItWbJE9913X6qnAgDIcg5jjMn0JIYqHA7L5/MpFArx/hQAZKHBPo7z3X0AAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGu5Mz2Bc2GMkSSFw+EMzwQAcC5OP36ffjw/m6yM1PHjxyVJVVVVGZ4JAGA4urq65PP5zro+KyNVVlYmSTp06NCX7txXXTgcVlVVldra2uT1ejM9HWtxnAaH4zQ4HKfBMcaoq6tLlZWVX7pdVkbK6fzkrTSfz8cfwSB4vV6O0yBwnAaH4zQ4HKc/bzBPMjhxAgBgLSIFALBWVkbK4/Ho3nvvlcfjyfRUrMZxGhyO0+BwnAaH45RaDvPnzv8DACBDsvKZFADgq4FIAQCsRaQAANYiUgAAa2VlpB555BFNmDBBeXl5mj17tnbs2JHpKaXV66+/riuvvFKVlZVyOBx64YUXktYbY7RmzRqdf/75ys/PV21trd57772kbU6cOKHFixfL6/WqpKRES5cuVXd3dxr3YmStXbtWs2bNUnFxscrLy7Vw4UIdOHAgaZu+vj7V19drzJgxKioq0qJFi9Te3p60zaFDh7RgwQIVFBSovLxcd911lwYGBtK5KyNq3bp1mjZtWuKDp4FAQJs3b06s5xid2f333y+Hw6EVK1YklnGsRojJMuvXrze5ubnm3//9382+ffvMzTffbEpKSkx7e3ump5Y2r7zyivnRj35knn/+eSPJbNy4MWn9/fffb3w+n3nhhRfMH/7wB3PVVVeZiRMnmt7e3sQ2l19+uZk+fbp54403zO9+9ztz4YUXmuuvvz7NezJy6urqzOOPP2727t1r9uzZY37wgx+Y6upq093dndjm1ltvNVVVVaaxsdHs2rXLzJkzx3z7299OrB8YGDBTpkwxtbW1Zvfu3eaVV14xY8eONatXr87ELo2Il156yfznf/6neffdd82BAwfMP/3TP5mcnByzd+9eYwzH6Ex27NhhJkyYYKZNm2buuOOOxHKO1cjIukhdcsklpr6+PvFzLBYzlZWVZu3atRmcVeZ8PlLxeNxUVFSYn/3sZ4llnZ2dxuPxmGeeecYYY8zbb79tJJmdO3cmttm8ebNxOBzmo48+Stvc06mjo8NIMk1NTcaYT45JTk6O2bBhQ2Kb/fv3G0mmubnZGPPJPwacTqcJBoOJbdatW2e8Xq+JRCLp3YE0Ki0tNb/+9a85RmfQ1dVlJk2aZBoaGsx3v/vdRKQ4ViMnq17u6+/vV0tLi2praxPLnE6namtr1dzcnMGZ2ePgwYMKBoNJx8jn82n27NmJY9Tc3KySkhLNnDkzsU1tba2cTqe2b9+e9jmnQygUkvT/v5y4paVF0Wg06ThNnjxZ1dXVScdp6tSp8vv9iW3q6uoUDoe1b9++NM4+PWKxmNavX6+enh4FAgGO0RnU19drwYIFScdE4u9pJGXVF8weO3ZMsVgs6f9kSfL7/XrnnXcyNCu7BINBSTrjMTq9LhgMqry8PGm92+1WWVlZYpvRJB6Pa8WKFbr00ks1ZcoUSZ8cg9zcXJWUlCRt+/njdKbjeHrdaNHa2qpAIKC+vj4VFRVp48aNqqmp0Z49ezhGn7F+/Xq9+eab2rlz5xfW8fc0crIqUsC5qK+v1969e/X73/8+01Ox0kUXXaQ9e/YoFArpueee05IlS9TU1JTpaVmlra1Nd9xxhxoaGpSXl5fp6XylZNXLfWPHjpXL5frCGTPt7e2qqKjI0Kzscvo4fNkxqqioUEdHR9L6gYEBnThxYtQdx+XLl2vTpk3aunWrxo0bl1heUVGh/v5+dXZ2Jm3/+eN0puN4et1okZubqwsvvFAzZszQ2rVrNX36dP3iF7/gGH1GS0uLOjo69K1vfUtut1tut1tNTU16+OGH5Xa75ff7OVYjJKsilZubqxkzZqixsTGxLB6Pq7GxUYFAIIMzs8fEiRNVUVGRdIzC4bC2b9+eOEaBQECdnZ1qaWlJbLNlyxbF43HNnj077XMeCcYYLV++XBs3btSWLVs0ceLEpPUzZsxQTk5O0nE6cOCADh06lHScWltbk4Le0NAgr9ermpqa9OxIBsTjcUUiEY7RZ8ydO1etra3as2dP4jZz5kwtXrw48b85ViMk02duDNX69euNx+MxTzzxhHn77bfNLbfcYkpKSpLOmBnturq6zO7du83u3buNJPMv//IvZvfu3ebDDz80xnxyCnpJSYl58cUXzVtvvWWuvvrqM56CfvHFF5vt27eb3//+92bSpEmj6hT0ZcuWGZ/PZ37729+ao0ePJm6nTp1KbHPrrbea6upqs2XLFrNr1y4TCARMIBBIrD99yvC8efPMnj17zKuvvmrOO++8UXXK8KpVq0xTU5M5ePCgeeutt8yqVauMw+Ew//Vf/2WM4Rh9mc+e3WcMx2qkZF2kjDHmX//1X011dbXJzc01l1xyiXnjjTcyPaW02rp1q5H0hduSJUuMMZ+chn7PPfcYv99vPB6PmTt3rjlw4EDSGMePHzfXX3+9KSoqMl6v19x4442mq6srA3szMs50fCSZxx9/PLFNb2+vue2220xpaakpKCgwP/zhD83Ro0eTxvnggw/M/PnzTX5+vhk7dqz5h3/4BxONRtO8NyPnpptuMuPHjze5ubnmvPPOM3Pnzk0EyhiO0Zf5fKQ4ViODS3UAAKyVVe9JAQC+WogUAMBaRAoAYC0iBQCwFpECAFiLSAEArEWkAADWIlIAAGsRKQCAtYgUAMBaRAoAYC0iBQCw1v8DalBShFCCjZYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "-7.952800418537099"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "student.test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第9章-策略梯度算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
